#!/bin/bash

set -o errexit
set -o xtrace

test_dir=$(realpath $(dirname $0))
. "${test_dir}"/../functions
cluster="self-healing-advanced"

if version_gt "1.19"; then
	echo "Kubernetes version is 1.19+ so skipping this test because of pumba support."
	exit 0
fi

crash_cluster_with_kubectl() {
    # run chaos for Pods
    for _ in $(seq 1 5); do
        kubectl_bin delete pod self-healing-advanced-pxc-0 self-healing-advanced-pxc-1 self-healing-advanced-pxc-2 --grace-period=0 --force || :
    done

    sleep 60

    for i in $(seq 0 2);do
        wait_crash_pod self-healing-advanced-pxc-$i
    done
}

pumba_all_pods() {
    local action=$1

    # run chaos for Pod
    for _ in $(seq 1 5); do
        run_pumba "$action" "re2:$cluster-pxc-[0-9]+_" || :
    done

    sleep 60

    for i in $(seq 0 2);do
        wait_crash_pod self-healing-advanced-pxc-$i
    done
}

check_if_cluster_restored(){
    desc 'check data consistency'
    compare_mysql_cmd "select-1" "SELECT * from myApp.myApp;" "-h $cluster-pxc-0.$cluster-pxc -uroot -proot_password"
    compare_mysql_cmd "select-1" "SELECT * from myApp.myApp;" "-h $cluster-pxc-1.$cluster-pxc -uroot -proot_password"
    compare_mysql_cmd "select-1" "SELECT * from myApp.myApp;" "-h $cluster-pxc-2.$cluster-pxc -uroot -proot_password"
}

main() {
    create_infra $namespace

    kubectl_bin apply -f "$test_dir/conf/pumba.yml"

    desc 'start cluster'
    spinup_pxc "$cluster" "$test_dir/conf/$cluster.yml"

    desc 'delete all PXC pods and check full crash'
    crash_cluster_with_kubectl

    cat_config "$test_dir/conf/$cluster.yml" \
        | sed -e 's/autoRecovery: false/autoRecovery: true/' \
        | kubectl_bin apply -f-

    wait_for_running $cluster-pxc 3
    sleep 240

    desc 'check if full cluster crash repaired'
    check_if_cluster_restored


    desc 'chaos pods with pumba'
    pumba_all_pods kill    

    wait_for_running $cluster-pxc 3
    sleep 240
    desc 'check if full cluster crash repaired'
    check_if_cluster_restored

    destroy "$namespace"
}

main
